package hotel.compare.oversea.service.impl;

import hotel.base.oversea.service.impl.BaseServiceImpl;
import hotel.base.oversea.utils.DateUtil;
import hotel.base.oversea.utils.StringUtil;
import hotel.compare.oversea.config.service.RedisService;
import hotel.compare.oversea.entity.master.SpiderHotelLog;
import hotel.compare.oversea.mapper.master.CompareHotelMapper;
import hotel.compare.oversea.mapper.master.CompareHotelSpiderMapper;
import hotel.compare.oversea.mapper.master.SpiderHotelLogMapper;
import hotel.compare.oversea.service.SpiderHotelLogService;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.HttpEntity;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.util.ObjectUtils;

import java.io.IOException;
import java.util.*;
import java.util.concurrent.*;


/**
 * 爬取携程酒店记录
 *
 * @author Zhang·JZ
 * @date 2020-05-28
 */
@Slf4j
@Service
public class SpiderHotelLogServiceImpl extends BaseServiceImpl<SpiderHotelLog> implements SpiderHotelLogService {

    @Autowired
    private SpiderHotelLogMapper spiderHotelLogMapper;

    @Autowired
    private CompareHotelMapper compareHotelMapper;

    @Autowired
    private CompareHotelSpiderMapper compareHotelSpiderMapper;

    @Autowired
    private RedisService redisService;

    //统计当前成功爬取条数
    private String loadKey = "spider_succ_hotel_load_num_key";

    //统计当前爬取条数
    private String loadSpiderKey = "spider_hotel_load_num_key";


    //统计当前爬取百分比
    private String loadPer = "spider_hotel_load_per_key";


    //返回内容
    private String loadMsg = "spider_hotel_load_msg_key";

    //成功已爬取条数
    private int spiderSuccCount = 0;

    //已爬取条数
    private int spiderCount = 0;
    //需要爬取总数
    private int totalCount = 0;

    //存储对应携程酒店房型
    private Map hotelRoomMap = new HashMap();


    public List<SpiderHotelLog> getSpiderHotelIdList(){
        List<SpiderHotelLog> spiderHotelLogList = new ArrayList<>();

        String hotelIds = "65113796,65113802,65272916,66073228,66073247,66073294,66073346,66730705,66779933,66841056,66869515,66876517,66876522,66876529,66876530,66876544,66876555,66876595,66876636,66876677,66935114,66935115,66935117,66935119,66935123,66935135,66935150,66935151,66935153,66935154,66935157,66935162,66935188,66935190,66935193,66990550,66990552,66990553,66990554,66990555,66990557,66990558,66990612,66990619,66990620,66990621,66990622,66990623,66990624,66990628,66990629,66990630,66990631,66990632,66990633,66990634,66990635,66990636,66990637,66990639,66990640,66990643,66990644,66990647,66990650,66990653,66990655,66990656,66990658,66991218,66991219,66991336,67053974,67053979,67053983,67053986,67053997,67054004,67054006,67054007,67054008,67054010,67054015,67167280,67167281,67167282,67167283,67167285,67223770,67224023,67226621,67226623,67226625,67226626,67226627,67226628,67226633,67226634,67226635,67226637,67226639,67226642,67226643,67226649,67226650,67226652,67226653,67226656,67226658,67226661,67226662,67226664,67226665,67226668,67226669,67226685,67226687,67226689,67226690,67226694,67226696,67226698,67226700,67226701,67226702,67226703,67226705,67226706,67226709,67226710,67226711,67226714,67226716,67226718,67226719,67226734,67226742,67226744,67226745,67226746,67226748,67226749,67226750,67226752,67226753,67226754,67226755,67226757,67226758,67310249,67310250,67310251,67310252,67310254,67310255,67310256,67310257,67310258,67310259,67310260,67310261,67310262,67310263,67310264,67310265,67310266,67310268,67310269,67310270,67314477,67314478,67314479,67314480,67314481,67314482,67314483,67314484,67314485,67314488,67314490,67314491,67314493,67314496,67314499,67314502,67314503,67314504,67314505,67314506,67314507,67314508,67314509,67314511,67314514,67314516,67314517,67314518,67314519,67314520,67314522,67314523,67314524,67314525,67314526,67314530,67314532,67314533,67314536,67314537,67314538,67314539,67316125,67316128,67316129,67316132,67316134,67316137,67316140,67316142,67316146,67316169,67316171,67316177,67316215,67316259,67316299,67316347,67316405,67316440,67316442,67316451,67316468,67316469,67316477,67316482,67316502,67316513,67316514,67316516,66935124,66990551,66990652,67053980,67167284,67223769,67226622,67226638,67226640,67226641,67226644,67226712,67226715,67314497,67316139,67316148,67316150,67316151,67316153,67316155,67316158,67316160,67316162,67316164,67316166,67316172,67316174,67316176,67316178,67316182,67316185,67316186,67316187,67316189,67316190,67316192,67316195,67316198,67316203,67316206,67316231,67316245,67316281,67316312,67316326,67316361,67316381,67316393,67316395,67316397,67316400,67316402,67316409,67316410,67316414,67316416,67316419,67316420,67316424,67316426,67316429,67316431,67316432,67316434,67316435,67316437,67316446,67316450,67316453,67316455,67316458,67316461,67316462,67316466,67316471,67316474,67316479,67316481,67316484,67316485,67316488,67316491,67316495,67316497,67316499,67316500,67316503,67316506,67316507,67316510,67316511,67316512,67316517,67316518,67619837,67619554,67620121,67619675,67620028,67619082,67620303,67620123,67619104,67620029,67620580,67619555,67620305,67620124,67620031,67621559,67619130,67619557,67053995,66990646,67619132,67619681,67621560,67620125,67620033,67619134,67619841,67620581,67620126,67619683,67620307,67621561,67621463,67620035,67619135,67621562,67620582,67620310,66990648,67619686,67620037,67621465,67619137,67620583,67619842,67619687,67619564,67620130,67621466,67620584,67619140,67621468,67620312,67619844,67619565,67621563,67620586,67054003,67619721,67620313,67621564,67620587,67620046,67621475,67619847,66990561,67619154,67620316,67620588,67620047,67619848,67620317,66990562,67620589,67621565,67621476,67620318,67619850,67054012,67620591,67619781,67620321,67619568,67620049,67621479,67620595,67619782,67620322,67619852,67620050,67620620,67619785,67619570,67621480,67619166,67620652,67621482,67620140,67620681,67619788,67621483,67619853,67619572,67620141,65113730,67619225,67620052,67619789,67621484,67620325,67619573,67620142,67619790,67621485,67619856,67620327,67620696,67621486,67619858,67620328,67619791,67619260,67620054,66990613,67621487,67620698,67619262,67620330,67619582,67620067,67621488,67619861,66990614,67619793,67619585,67621489,67620700,67619264,67619862,67619794,67620331,67621490,67619864,67619266,67621493,67619588,67620701,67619269,67620071,67621495,67619796,67618807,67619870,67620364,67620702,67226632,67620073,67619271,67621496,67620382,67619273,67620705,67621499,67620410,67618808,67619592,67620708,67619275,67620075,67620709,67619277,67619799,67620077,67619593,67620463,67618811,67619278,67620484,67618812,67621507,67619876,67619280,67620503,67620081,67618814,67621509,67619893,67619282,65113736,67619803,67620713,67620507,67621511,67619283,67618815,67620509,67621512,66990618,67620149,67619952,67620087,67618817,67619622,67620151,67620088,67618819,67621515,67619979,67619804,67620510,67620153,67620090,67620716,67618824,67619805,67621516,67619982,67620154,67620511,67619645,67621518,67620719,67618826,67619287,67620155,67619986,67619806,67620513,67620095,67618828,67621519,67619646,67619988,67619808,67620515,67618829,67620097,67619647,67620156,67619809,67621520,67619811,67620099,67620518,67619648,67621521,67619289,67618853,67621523,67619649,67619992,67620158,67619812,67618855,67621529,67620521,67620101,67618857,67621532,67619651,67620162,67619309,67224024,67619817,67621533,67618858,67619999,67619653,67619821,67620166,67618862,67621535,67619377,67620168,67620005,67621536,67619404,67620007,67619824,67619655,67621538,67619432,67620008,67620107,67619825,67618864,67621539,67620530,67619657,67620009,67620108,67619469,67618866,67621540,67619826,67619659,67620218,67620012,67620110,67618882,67621541,67620013,67620248,67618912,67619524,67620113,67619660,67620014,67619829,67619541,67619661,67618943,67620015,67621545,67619542,67620296,67618965,67619662,67621546,67620017,67619543,67619831,67621548,67618967,67620018,67619832,67620553,67620115,67621549,67620019,67620300,67619665,67619546,67619833,67618971,67620116,67620569,67619548,67620021,67620117,67618974,67620571,67621552,67619667,67620024,67619549,67618992,67619669,67620573,67620120,67619017,67621555,67619552,67619834,67620576,67620025,66990642,67619670,67619553,67621556,67620026,67619671";
        String[] hotelIdsArr = hotelIds.split(",");
        for(int i=0;i<hotelIdsArr.length;i++){
            SpiderHotelLog spiderHotelLog = new SpiderHotelLog();
            spiderHotelLog.setId(StringUtil.getUUID());
            spiderHotelLog.setSwitchsHotelId(Integer.valueOf(hotelIdsArr[i]));
            spiderHotelLog.setCreateTime(DateUtil.getToday());
            spiderHotelLog.setModifyTime(new Date());
            spiderHotelLog.setState(0);
            spiderHotelLog.setIsAgent(0);//默认没有代理

            spiderHotelLogList.add(spiderHotelLog);
        }

        if(!spiderHotelLogList.isEmpty() && spiderHotelLogList.size()>0){
            int insInt = spiderHotelLogMapper.insertList(spiderHotelLogList);
            log.info("新增成功--->>> {} 条数据",insInt);
        }

        return spiderHotelLogList;
    }

    /**
     * 爬取已创建的就带你，判断携程酒店是否有代理
     * @return
     */
    @Override
    public boolean tripSpiderHotelAgentByIsCreate(List<SpiderHotelLog> spiderHotelLogList) throws InterruptedException, ExecutionException {
        if(redisService.exists(loadKey) && redisService.exists(loadSpiderKey)){
            redisService.remove(loadKey);
            redisService.remove(loadSpiderKey);
        }
        //开始时间
        long start = System.currentTimeMillis();
        //当天
//        String nowDate = DateUtil.getToday();
//
//        SpiderHotelLog spiderHotelLog = new SpiderHotelLog();
//        spiderHotelLog.setCreateTime(nowDate);
//        List<SpiderHotelLog> spiderHotelLogList = spiderHotelLogMapper.select(spiderHotelLog);//总酒店数量

        //test
//        List<SpiderHotelLog> spiderHotelLogList = new ArrayList<>();
//        spiderHotelLogList = getSpiderHotelIdList();

        StringBuffer ret = new StringBuffer();
        // 总数
        int dataSize = spiderHotelLogList.size();
        totalCount = dataSize;
        // 线程数
        int threadNum = 20;
        //大于10万，则用20个线程
        if(dataSize>100000){
            threadNum=30;
        }

        // 每500条数据开启一条线程
        int num = 0;
        if((dataSize%threadNum) > 0){
            num = 1;
        }
        int threadSize = dataSize/threadNum + num;
        // 定义标记,过滤threadNum为整数
        boolean special = dataSize % threadSize == 0;

        // 创建一个线程池
        ExecutorService exec = Executors.newFixedThreadPool(threadNum);
        // 定义一个任务集合
        List<Callable<Integer>> tasks = new ArrayList<Callable<Integer>>();
        Callable<Integer> task = null;
        List<SpiderHotelLog> cutList = new ArrayList<>();

        //如果数量小直接执行
        if(dataSize<= threadNum){
            final String taskNum = String.valueOf(1);
            task = new Callable<Integer>() {
                @Override
                public Integer call() throws Exception {

                    Thread.sleep(10);
                    checkCompareListSpider(taskNum,spiderHotelLogList);
                    return 1;
                }
            };
            // 这里提交的任务容器列表和返回的Future列表存在顺序对应的关系
            tasks.add(task);
        }else {
            // 确定每条线程的数据
            for (int i = 0; i < threadNum; i++) {
                if (i == threadNum - 1) {
                    if (special) {
                        break;
                    }
                    cutList = spiderHotelLogList.subList(threadSize * i, dataSize);
                } else {
                    cutList = spiderHotelLogList.subList(threadSize * i, threadSize * (i + 1));
                }
                final List<SpiderHotelLog> listStr = cutList;
                final String taskNum = String.valueOf(i + 1);
//            final List<CompareHotel> listStr = compareHotelList; //开启单线程
                task = new Callable<Integer>() {

                    @Override
                    public Integer call() throws Exception {

                        Thread.sleep(10);
                        checkCompareListSpider(taskNum, listStr);
                        return 1;
                    }
                };
                // 这里提交的任务容器列表和返回的Future列表存在顺序对应的关系
                tasks.add(task);
            }
        }

        List<Future<Integer>> results = exec.invokeAll(tasks);

        for (Future<Integer> future : results) {
            log.info("future.get()>>>" + future.get().toString());
            ret.append(future.get());
        }

        // 关闭线程池
        exec.shutdown();
        log.info("线程任务执行结束");
        long end = System.currentTimeMillis();
        log.info("爬虫时长为：{}",(end-start));
        return true;
    }

    public void checkCompareListSpider(String taskNum,List<SpiderHotelLog> spiderHotelLogList){
        if(!ObjectUtils.isEmpty(spiderHotelLogList) && spiderHotelLogList.size()>0){
            spiderHotelLogList.forEach(spiderHotelLog -> {
//                log.info("spiderHotelLog ---> {}", JacksonUtil.objectToJson(spiderHotelLog));
                //如果存在代理状态null 或者  0 的修改为有代理；原来是有代理则不做修改
                if(ObjectUtils.isEmpty(spiderHotelLog.getIsAgent()) || spiderHotelLog.getIsAgent() == 0){
                    String html = getHotelBySpider(String.valueOf(spiderHotelLog.getSwitchsHotelId()));
                    int updInt = 0;
                    if(html.indexOf("label_onsale_blue") != -1){
                        updInt = spiderHotelLogMapper.updateSpiderHotelLogIsAgent(spiderHotelLog.getId());
//                        if(updInt > 0){
//                            updInt = compareHotelSpiderMapper.updateCompareHotelSpiderIsAgent(String.valueOf(spiderHotelLog.getSwitchsHotelId()));

                            log.info("-------------酒店{} 售卖房型中存在代理房型，已更新--------------",spiderHotelLog.getSwitchsHotelId());
//                        }
                    }
                    synchronized (this) {
                        log.info("synchronized--spiderCount-->>> {}",spiderCount);
                        String tempStr = "";
                        //已经爬取酒店的数量
                        if(redisService.exists(loadSpiderKey)){
                            spiderCount = (int) redisService.get(loadSpiderKey);

                            spiderCount = spiderCount + 1;
                            redisService.set(loadSpiderKey,spiderCount);
                        }else{
                            spiderCount = spiderCount + 1;
                            redisService.set(loadSpiderKey,spiderCount);
                        }

                        tempStr = "爬取数量:"+spiderCount;
                        //爬取有代理的酒店数量
                        if(updInt > 0) {
                            if (redisService.exists(loadKey)) {
                                spiderSuccCount = (int) redisService.get(loadKey);
                                spiderSuccCount = spiderSuccCount + 1;
                                redisService.set(loadKey, spiderSuccCount);
                            }else{
                                spiderSuccCount = spiderSuccCount + 1;
                                redisService.set(loadKey, spiderSuccCount);
                            }
                        }

                        tempStr = tempStr+";有代理数量:"+spiderSuccCount;
                        if(!ObjectUtils.isEmpty(tempStr)){
                            redisService.set(loadMsg,tempStr);
                        }
                    }

                }else {
                    log.info("============当前酒店 {} 存在代理不需要爬取============",spiderHotelLog.getSwitchsHotelId());
                }


//                Document document = Jsoup.parse(html);
//                //判斷是否有代理
//                Elements postItems = document.getElementsByClass("label_onsale_blue");
//                if(!ObjectUtils.isEmpty(postItems)){
//                    //如果原来是有代理则不做修改，否则修改
//                    if(!ObjectUtils.isEmpty(spiderHotelLog.getIsAgent()) && spiderHotelLog.getIsAgent() == 0){
//                        spiderHotelLog spiderHotelLogAgent = new spiderHotelLog();
//                        spiderHotelLogAgent.setId(spiderHotelLog.getId());
//                        spiderHotelLogAgent.setIsAgent(1);//有代理
//                        int updInt = spiderHotelLogMapper.updateByPrimaryKey(spiderHotelLogAgent);
//                        if(updInt > 0){
//                            log.info("酒店{} 售卖房型中存在代理房型",spiderHotelLog.getSwitchsHotelId());
//                        }
//                    }
//                }
            });
        }
    }

    /**
     * 爬取携程酒店信息
     * @param hotelId
     * @return
     */
    public String getHotelBySpider(String hotelId) {
        log.info("<<<<<<<<<<<<开始爬取酒店id 为： {} >>>>>>>>>>>",hotelId);
        String url = "http://hotels.ctrip.com/Domestic/tool/AjaxHote1RoomListForDetai1.aspx?hotel="+ hotelId;
        CloseableHttpClient httpClient = null;
        CloseableHttpResponse response = null;
        String result = "";
        try {
            // 通过址默认配置创建一个httpClient实例
            httpClient = HttpClients.createDefault();
            // 创建httpGet远程连接实例
            HttpGet httpGet = new HttpGet(url);
            // 传输的类型
            httpGet.addHeader("cookie", "nfes_isSupportWebP=1; magicid=XPzkJHQ5LzTp3RWjeLkHRy6a72G81VLaWEsQk7FAEPzBaLSQv4yIN4/TI76Mhhde; _abtest_userid=e285f6c3-33b8-4877-ac74-49a22fcef8cd; hoteluuid=GPDNup39QtsOCadM; _RSG=0zqTvkAod2EQuXuknBtMG9; _RDG=288fe77d7e4db22565046a3c0b7705d03d; _RGUID=e0a7c8ab-b977-4f40-9416-dcbc6ee21e1a; MKT_CKID=1590660817624.fsqmi.fdmh; _ga=GA1.2.43875582.1590660818; librauuid=u3U0ptmP9zzU0tgdCb; __utma=1.43875582.1590660818.1595405355.1595405355.1; __utmz=1.1595405355.1.1.utmcsr=baidu|utmccn=(organic)|utmcmd=organic; UUID=FC7634B2022346B2B0549A7A0E10569A; IsPersonalizedLogin=F; ASP.NET_SessionId=e5ndmqrqlebnyqyc0ulnalqt; OID_ForOnlineHotel=15894554982802ubusu1596629676939102003; appFloatCnt=25; GUID=09031078412603118855; _gcl_dc=GCL.1598322724.CJjssZCotesCFQUZvAodIOoDWQ; manualclose=1; login_uid=AAE585922AA36B33F5EBDD08395BA989; login_type=6; cticket=05709ABD2460E38544E350D5ED95F75ED4B50CD9931AA76BEE1596878B070633; AHeadUserInfo=VipGrade=0&VipGradeName=%C6%D5%CD%A8%BB%E1%D4%B1&UserName=&NoReadMessageCount=0; ticket_ctrip=bJ9RlCHVwlu1ZjyusRi+ypZ7X2r4+yojKV4y12iWgxmNHtey2jmciPKSl6dh+KksUv+HFgAlX4bGD3i7CD/dIh5w6SCgo6k/sNeW5RFsqHJgAVzLFY6BlYcKWxBMzZng8Zd22FHL6yjfqBq+EdnOrxJDRp15hUKPi7KRaBf9yxrB7/SpsakAFJVYxwMUWaC/4GwZ6gpqwLPFpz7/7GO1tPPm4CeSdAXl+nWdGHq8nytQFK0MNf3UlZ4ZvT1aPr58oc4nQbpC7yfLpi80m9FDFFO/1biGGWvjqF4ae4AMwe8=; DUID=u=47DC1042E0B102490510B157CDE44914&v=0; IsNonUser=F; _RF1=106.122.243.247; _gid=GA1.2.1479578356.1598932568; MKT_CKID_LMT=1598932568116; Session=SmartLinkCode=U155952&SmartLinkKeyWord=&SmartLinkQuary=&SmartLinkHost=&SmartLinkLanguage=zh; Union=OUID=index&AllianceID=4897&SID=155952&SourceID=&createtime=1598943675&Expires=1599548475102; MKT_OrderClick=ASID=4897155952&AID=4897&CSID=155952&OUID=index&CT=1598943675106&CURL=https%3A%2F%2Fwww.ctrip.com%2F%3Fsid%3D155952%26allianceid%3D4897%26ouid%3Dindex&VAL={\"pc_vid\":\"1589455498280.2ubusu\"}; HotelCityID=25split%E5%8E%A6%E9%97%A8splitXiamensplit2020-9-1split2020-09-02split0; HotelDomesticVisitedHotels1=15899701=0,0,4.7,3436,/200k1f000001glax070F5.jpg,&1028029=0,0,0,0,/t1/hotel/1029000/1028085/9bd985b2e8d945b7b764454e88ad97b7.jpg,&848702=0,0,4.3,1546,/20030800000032619153C.jpg,&35940527=0,0,4.8,360,/200v1c000001cz8lk0724.jpg,&1628452=0,0,4.1,138,/20010g0000007vysyC609.jpg,; MKT_Pagesource=PC; _jzqco=%7C%7C%7C%7C1598932568238%7C1.1652538683.1590660817618.1598964660560.1598964980508.1598964660560.1598964980508.undefined.0.0.91.91; __zpspc=9.41.1598964660.1598964980.2%232%7Cwww.baidu.com%7C%7C%7C%25E6%2590%25BA%25E7%25A8%258B%25E7%25BD%2591%7C%23; _bfi=p1%3D102003%26p2%3D102003%26v1%3D4169%26v2%3D4168; _bfa=1.1589455498280.2ubusu.1.1598964652692.1598964657593.298.4170.212094; _bfs=1.3; hotelhst=50349362");
            httpGet.addHeader("referer", "http://hotels.ctrip.com/hotel/"+hotelId+".html");
            httpGet.addHeader("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50");

            // 设置配置请求参数
            RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(35000)// 连接主机服务超时时间
                    .setConnectionRequestTimeout(35000)// 请求超时时间
                    .setSocketTimeout(60000)// 数据读取超时时间
                    .build();
            // 为httpGet实例设置配置
            httpGet.setConfig(requestConfig);
            // 执行get请求得到返回对象
            response = httpClient.execute(httpGet);
            // 通过返回对象获取返回数据
            HttpEntity entity = response.getEntity();
            // 通过EntityUtils中的toString方法将结果转换为字符串
            result = EntityUtils.toString(entity);
        } catch (ClientProtocolException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            // 关闭资源
            if (null != response) {
                try {
                    response.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
            if (null != httpClient) {
                try {
                    httpClient.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
        return result;
    }



    /**
     * 爬取已创建的就带你，判断携程酒店是否有代理
     * @return
     */
    @Override
    public boolean tripSpiderHotelAgentByIsCreateOld(List<SpiderHotelLog> spiderHotelLogList) throws InterruptedException, ExecutionException {
        if(redisService.exists(loadKey) && redisService.exists(loadSpiderKey)){
            redisService.remove(loadKey);
            redisService.remove(loadSpiderKey);
        }
        //开始时间
        long start = System.currentTimeMillis();
        //当天
//        String nowDate = DateUtil.getToday();
//
//        SpiderHotelLog spiderHotelLog = new SpiderHotelLog();
//        spiderHotelLog.setCreateTime(nowDate);
//        List<SpiderHotelLog> spiderHotelLogList = spiderHotelLogMapper.select(spiderHotelLog);//总酒店数量

        //test
//        List<SpiderHotelLog> spiderHotelLogList = new ArrayList<>();
//        spiderHotelLogList = getSpiderHotelIdList();

        StringBuffer ret = new StringBuffer();
        // 总数
        int dataSize = spiderHotelLogList.size();
        totalCount = dataSize;
        // 线程数
        int threadNum = 1;
        //大于10万，则用20个线程
        if(dataSize>100000){
            threadNum=30;
        }

        // 每500条数据开启一条线程
        int num = 0;
        if((dataSize%threadNum) > 0){
            num = 1;
        }
        int threadSize = dataSize/threadNum + num;
        // 定义标记,过滤threadNum为整数
        boolean special = dataSize % threadSize == 0;

        // 创建一个线程池
        ExecutorService exec = Executors.newFixedThreadPool(threadNum);
        // 定义一个任务集合
        List<Callable<Integer>> tasks = new ArrayList<Callable<Integer>>();
        Callable<Integer> task = null;
        List<SpiderHotelLog> cutList = new ArrayList<>();

        //如果数量小直接执行
        if(dataSize<= threadNum){
            final String taskNum = String.valueOf(1);
            task = new Callable<Integer>() {
                @Override
                public Integer call() throws Exception {

                    Thread.sleep(10);
                    checkCompareListSpider(taskNum,spiderHotelLogList);
                    return 1;
                }
            };
            // 这里提交的任务容器列表和返回的Future列表存在顺序对应的关系
            tasks.add(task);
        }else {
            // 确定每条线程的数据
            for (int i = 0; i < threadNum; i++) {
                if (i == threadNum - 1) {
                    if (special) {
                        break;
                    }
                    cutList = spiderHotelLogList.subList(threadSize * i, dataSize);
                } else {
                    cutList = spiderHotelLogList.subList(threadSize * i, threadSize * (i + 1));
                }
                final List<SpiderHotelLog> listStr = cutList;
                final String taskNum = String.valueOf(i + 1);
//            final List<CompareHotel> listStr = compareHotelList; //开启单线程
                task = new Callable<Integer>() {

                    @Override
                    public Integer call() throws Exception {

                        Thread.sleep(10);
                        checkCompareListSpider(taskNum, listStr);
                        return 1;
                    }
                };
                // 这里提交的任务容器列表和返回的Future列表存在顺序对应的关系
                tasks.add(task);
            }
        }

        List<Future<Integer>> results = exec.invokeAll(tasks);

//        for (Future<Integer> future : results) {
//            log.info("future.get()>>>" + future.get().toString());
//            ret.append(future.get());
//        }

        // 关闭线程池
        exec.shutdown();
        log.info("线程任务执行结束");
        long end = System.currentTimeMillis();
        log.info("爬虫时长为：{}",(end-start));
        return true;
    }
}
